from sklearn.metrics import precision_score, recall_score, f1_score, accuracy_score, roc_auc_score

from data_loader.siamese_data_loader import SiameseDataLoader
from models.siamese_model import CNNSiameseModel, BiLSTMSiameseModel, CNNBiLSTMSiameseModel, SiameseCategoricalModel,\
    SiameseMixedCNNDistanceModel, SiameseMixedBiLSTMDistanceModel, SiameseMixedCNNBiLSTMDistanceModel,\
    SiameseMixedCategoricalModel, EnsembleModel, BiLSTMAttSiameseModel
from trainers.siamese_trainer import SiameseModelTrainer, SiameseMixedTrainer, EnsembleTrainer
from configs.siamese_config import Config
from utils.dirs import create_dirs
from utils.utils import post_processing, output_error_lines

import xgboost as xgb

import numpy as np
import os


def cnn_siamese(level):
    config = Config()
    config.exp_name = 'cnn_siamese_' + level
    config.tensorboard_log_dir = os.path.join('experiments', config.exp_name, 'logs')
    config.checkpoint_dir = os.path.join('experiments', config.exp_name, 'checkpoints')
    config.embedding_path = 'custom_data/%s_level' % level
    if level == 'word':
        config.max_len = config.max_len_word
    else:
        config.max_len = config.max_len_char

    create_dirs([config.tensorboard_log_dir, config.checkpoint_dir])

    print('Create the data generator.')
    data_loader = SiameseDataLoader(config,
                                    train_file='datasets/train.csv',
                                    valid_file='datasets/valid.csv',
                                    level=level)

    print('Create the model.')
    cnn_siamese_model = CNNSiameseModel(config, data_loader.get_vocabulary())

    print('Create the trainer')
    trainer = SiameseModelTrainer(cnn_siamese_model.model,
                                  data_loader.get_train_data(),
                                  data_loader.get_valid_data(),
                                  config)

    print('Start training the model.')
    trainer.train()

    print('Start evaluate the model.')
    best_weights = os.path.join('experiments', config.exp_name, 'checkpoints', config.exp_name + '.hdf5')
    cnn_siamese_model.load(best_weights)
    valid_x_a, valid_x_b, valid_y = data_loader.get_valid_data()
    valid_results = cnn_siamese_model.model.predict([valid_x_a, valid_x_b])
    valid_y_pred = [valid_result[0] < 0.6 for valid_result in valid_results]
    valid_y_true = valid_y
    precision = precision_score(valid_y_true, valid_y_pred)
    recall = recall_score(valid_y_true, valid_y_pred)
    f1 = f1_score(valid_y_true, valid_y_pred)
    accuracy = accuracy_score(valid_y_true, valid_y_pred)
    auc = roc_auc_score(valid_y_true, valid_y_pred)
    print '\n- **Evaluation results of CNN siamese %s model**\n' % level
    print 'Precision:', precision, '\n'
    print 'Recall:', recall, '\n'
    print 'F1:', f1, '\n'
    print 'Accuracy:', accuracy, '\n'
    print 'Auc:', auc, '\n'


def cnn_siamese_mixed():
    config = Config()
    config.exp_name = 'cnn_siamese_mixed'
    config.tensorboard_log_dir = os.path.join('experiments', config.exp_name, 'logs')
    config.checkpoint_dir = os.path.join('experiments', config.exp_name, 'checkpoints')
    config.embedding_path_word = 'custom_data/word_level'
    config.embedding_path_char = 'custom_data/char_level'

    create_dirs([config.tensorboard_log_dir, config.checkpoint_dir])

    print('Create the data generator.')
    data_loader_word = SiameseDataLoader(config,
                                         train_file='datasets/train.csv',
                                         valid_file='datasets/valid.csv',
                                         level='word')

    data_loader_char = SiameseDataLoader(config,
                                         train_file='datasets/train.csv',
                                         valid_file='datasets/valid.csv',
                                         level='char')

    print('Create the model.')
    cnn_siamese_mixed_model = SiameseMixedCNNDistanceModel(config,
                                                           data_loader_word.get_vocabulary(),
                                                           data_loader_char.get_vocabulary())

    print('Create the trainer')
    trainer = SiameseMixedTrainer(cnn_siamese_mixed_model.model,
                                  data_loader_word.get_train_data(),
                                  data_loader_word.get_valid_data(),
                                  data_loader_char.get_train_data(),
                                  data_loader_char.get_valid_data(),
                                  config)

    print('Start training the model.')
    trainer.train()

    print('Start evaluate the model.')
    best_weights = os.path.join('experiments', config.exp_name, 'checkpoints', config.exp_name + '.hdf5')
    cnn_siamese_mixed_model.load(best_weights)
    valid_x_a_word, valid_x_b_word, valid_y_word = data_loader_word.get_valid_data()
    valid_x_a_char, valid_x_b_char, valid_y_char = data_loader_char.get_valid_data()
    assert valid_y_word == valid_y_char
    valid_results = cnn_siamese_mixed_model.model.predict([valid_x_a_word, valid_x_b_word,
                                                           valid_x_a_char, valid_x_b_char])
    valid_y_pred = [valid_result[0] < 0.6 for valid_result in valid_results]
    valid_y_true = valid_y_word
    precision = precision_score(valid_y_true, valid_y_pred)
    recall = recall_score(valid_y_true, valid_y_pred)
    f1 = f1_score(valid_y_true, valid_y_pred)
    accuracy = accuracy_score(valid_y_true, valid_y_pred)
    auc = roc_auc_score(valid_y_true, valid_y_pred)
    print '\n- **Evaluation results of CNN siamese mixed model\n'
    print 'Precision:', precision, '\n'
    print 'Recall:', recall, '\n'
    print 'F1:', f1, '\n'
    print 'Accuracy:', accuracy, '\n'
    print 'Auc:', auc, '\n'


def bilstm_siamese(level):
    config = Config()
    config.exp_name = 'bilstm_siamese_' + level
    config.tensorboard_log_dir = os.path.join('experiments', config.exp_name, 'logs')
    config.checkpoint_dir = os.path.join('experiments', config.exp_name, 'checkpoints')
    config.embedding_path = 'custom_data/%s_level' % level
    if level == 'word':
        config.max_len = config.max_len_word
    else:
        config.max_len = config.max_len_char

    create_dirs([config.tensorboard_log_dir, config.checkpoint_dir])

    print('Create the data generator.')
    data_loader = SiameseDataLoader(config,
                                    train_file='datasets/train.csv',
                                    valid_file='datasets/valid.csv',
                                    level=level)

    print('Create the model.')
    bilstm_siamese_model = BiLSTMSiameseModel(config, data_loader.get_vocabulary())

    print('Create the trainer')
    trainer = SiameseModelTrainer(bilstm_siamese_model.model,
                                  data_loader.get_train_data(),
                                  data_loader.get_valid_data(),
                                  config)

    print('Start training the model.')
    trainer.train()

    print('Start evaluate the model.')
    best_weights = os.path.join('experiments', config.exp_name, 'checkpoints', config.exp_name + '.hdf5')
    bilstm_siamese_model.load(best_weights)
    valid_x_a, valid_x_b, valid_y = data_loader.get_valid_data()
    valid_results = bilstm_siamese_model.model.predict([valid_x_a, valid_x_b])
    valid_y_pred = [valid_result[0] < 0.65 for valid_result in valid_results]
    valid_y_true = valid_y
    precision = precision_score(valid_y_true, valid_y_pred)
    recall = recall_score(valid_y_true, valid_y_pred)
    f1 = f1_score(valid_y_true, valid_y_pred)
    accuracy = accuracy_score(valid_y_true, valid_y_pred)
    auc = roc_auc_score(valid_y_true, valid_y_pred)
    print '\n- **Evaluation results of BiLSTM siamese %s model**\n' % level
    print 'Precision:', precision, '\n'
    print 'Recall:', recall, '\n'
    print 'F1:', f1, '\n'
    print 'Accuracy:', accuracy, '\n'
    print 'Auc:', auc, '\n'

    output_error_lines(valid_y_pred, valid_y_true, valid_results, data_loader)

    valid_y_pred = post_processing(valid_y_pred, data_loader)
    precision = precision_score(valid_y_true, valid_y_pred)
    recall = recall_score(valid_y_true, valid_y_pred)
    f1 = f1_score(valid_y_true, valid_y_pred)
    accuracy = accuracy_score(valid_y_true, valid_y_pred)
    auc = roc_auc_score(valid_y_true, valid_y_pred)
    print '\n- **Evaluation results of BiLSTM siamese %s model (post processing)**\n' % level
    print 'Precision:', precision, '\n'
    print 'Recall:', recall, '\n'
    print 'F1:', f1, '\n'
    print 'Accuracy:', accuracy, '\n'
    print 'Auc:', auc, '\n'


def bilstm_siamese_mixed():
    config = Config()
    config.exp_name = 'bilstm_siamese_mixed'
    config.tensorboard_log_dir = os.path.join('experiments', config.exp_name, 'logs')
    config.checkpoint_dir = os.path.join('experiments', config.exp_name, 'checkpoints')
    config.embedding_path_word = 'custom_data/word_level'
    config.embedding_path_char = 'custom_data/char_level'

    create_dirs([config.tensorboard_log_dir, config.checkpoint_dir])

    print('Create the data generator.')
    data_loader_word = SiameseDataLoader(config,
                                         train_file='datasets/train.csv',
                                         valid_file='datasets/valid.csv',
                                         level='word')

    data_loader_char = SiameseDataLoader(config,
                                         train_file='datasets/train.csv',
                                         valid_file='datasets/valid.csv',
                                         level='char')

    print('Create the model.')
    bilstm_siamese_mixed_model = SiameseMixedBiLSTMDistanceModel(config,
                                                                 data_loader_word.get_vocabulary(),
                                                                 data_loader_char.get_vocabulary())

    print('Create the trainer')
    trainer = SiameseMixedTrainer(bilstm_siamese_mixed_model.model,
                                  data_loader_word.get_train_data(),
                                  data_loader_word.get_valid_data(),
                                  data_loader_char.get_train_data(),
                                  data_loader_char.get_valid_data(),
                                  config)

    print('Start training the model.')
    trainer.train()

    print('Start evaluate the model.')
    best_weights = os.path.join('experiments', config.exp_name, 'checkpoints', config.exp_name + '.hdf5')
    bilstm_siamese_mixed_model.load(best_weights)
    valid_x_a_word, valid_x_b_word, valid_y_word = data_loader_word.get_valid_data()
    valid_x_a_char, valid_x_b_char, valid_y_char = data_loader_char.get_valid_data()
    assert valid_y_word == valid_y_char
    valid_results = bilstm_siamese_mixed_model.model.predict([valid_x_a_word, valid_x_b_word,
                                                              valid_x_a_char, valid_x_b_char])
    valid_y_pred = [valid_result[0] < 0.6 for valid_result in valid_results]
    valid_y_true = valid_y_word
    precision = precision_score(valid_y_true, valid_y_pred)
    recall = recall_score(valid_y_true, valid_y_pred)
    f1 = f1_score(valid_y_true, valid_y_pred)
    accuracy = accuracy_score(valid_y_true, valid_y_pred)
    auc = roc_auc_score(valid_y_true, valid_y_pred)
    print '\n- **Evaluation results of BiLSTM siamese mixed model\n'
    print 'Precision:', precision, '\n'
    print 'Recall:', recall, '\n'
    print 'F1:', f1, '\n'
    print 'Accuracy:', accuracy, '\n'
    print 'Auc:', auc, '\n'


def cnn_bilstm_siamese(level):
    config = Config()
    config.exp_name = 'cnn_bilstm_siamese_' + level
    config.tensorboard_log_dir = os.path.join('experiments', config.exp_name, 'logs')
    config.checkpoint_dir = os.path.join('experiments', config.exp_name, 'checkpoints')
    config.embedding_path = 'custom_data/%s_level' % level
    if level == 'word':
        config.max_len = config.max_len_word
    else:
        config.max_len = config.max_len_char

    create_dirs([config.tensorboard_log_dir, config.checkpoint_dir])

    print('Create the data generator.')
    data_loader = SiameseDataLoader(config,
                                    train_file='datasets/train.csv',
                                    valid_file='datasets/valid.csv',
                                    level=level)

    print('Create the model.')
    cnn_bilstm_siamese_model = CNNBiLSTMSiameseModel(config, data_loader.get_vocabulary())

    print('Create the trainer')
    trainer = SiameseModelTrainer(cnn_bilstm_siamese_model.model,
                                  data_loader.get_train_data(),
                                  data_loader.get_valid_data(),
                                  config)

    print('Start training the model.')
    trainer.train()

    print('Start evaluate the model.')
    best_weights = os.path.join('experiments', config.exp_name, 'checkpoints', config.exp_name + '.hdf5')
    cnn_bilstm_siamese_model.load(best_weights)
    valid_x_a, valid_x_b, valid_y = data_loader.get_valid_data()
    valid_results = cnn_bilstm_siamese_model.model.predict([valid_x_a, valid_x_b])
    valid_y_pred = [valid_result[0] < 0.6 for valid_result in valid_results]
    valid_y_true = valid_y
    precision = precision_score(valid_y_true, valid_y_pred)
    recall = recall_score(valid_y_true, valid_y_pred)
    f1 = f1_score(valid_y_true, valid_y_pred)
    accuracy = accuracy_score(valid_y_true, valid_y_pred)
    auc = roc_auc_score(valid_y_true, valid_y_pred)
    print '\n- **Evaluation results of CNN-BiLSTM siamese %s model**\n' % level
    print 'Precision:', precision, '\n'
    print 'Recall:', recall, '\n'
    print 'F1:', f1, '\n'
    print 'Accuracy:', accuracy, '\n'
    print 'Auc:', auc, '\n'


def cnn_bilstm_siamese_mixed():
    config = Config()
    config.exp_name = 'cnn_bilstm_siamese_mixed'
    config.tensorboard_log_dir = os.path.join('experiments', config.exp_name, 'logs')
    config.checkpoint_dir = os.path.join('experiments', config.exp_name, 'checkpoints')
    config.embedding_path_word = 'custom_data/word_level'
    config.embedding_path_char = 'custom_data/char_level'

    create_dirs([config.tensorboard_log_dir, config.checkpoint_dir])

    print('Create the data generator.')
    data_loader_word = SiameseDataLoader(config,
                                         train_file='datasets/train.csv',
                                         valid_file='datasets/valid.csv',
                                         level='word')

    data_loader_char = SiameseDataLoader(config,
                                         train_file='datasets/train.csv',
                                         valid_file='datasets/valid.csv',
                                         level='char')

    print('Create the model.')
    cnn_bilstm_siamese_mixed_model = SiameseMixedCNNBiLSTMDistanceModel(config,
                                                                        data_loader_word.get_vocabulary(),
                                                                        data_loader_char.get_vocabulary())

    print('Create the trainer')
    trainer = SiameseMixedTrainer(cnn_bilstm_siamese_mixed_model.model,
                                  data_loader_word.get_train_data(),
                                  data_loader_word.get_valid_data(),
                                  data_loader_char.get_train_data(),
                                  data_loader_char.get_valid_data(),
                                  config)

    print('Start training the model.')
    trainer.train()

    print('Start evaluate the model.')
    best_weights = os.path.join('experiments', config.exp_name, 'checkpoints', config.exp_name + '.hdf5')
    cnn_bilstm_siamese_mixed_model.load(best_weights)
    valid_x_a_word, valid_x_b_word, valid_y_word = data_loader_word.get_valid_data()
    valid_x_a_char, valid_x_b_char, valid_y_char = data_loader_char.get_valid_data()
    assert valid_y_word == valid_y_char
    valid_results = cnn_bilstm_siamese_mixed_model.model.predict([valid_x_a_word, valid_x_b_word,
                                                                  valid_x_a_char, valid_x_b_char])
    valid_y_pred = [valid_result[0] < 0.6 for valid_result in valid_results]
    valid_y_true = valid_y_word
    precision = precision_score(valid_y_true, valid_y_pred)
    recall = recall_score(valid_y_true, valid_y_pred)
    f1 = f1_score(valid_y_true, valid_y_pred)
    accuracy = accuracy_score(valid_y_true, valid_y_pred)
    auc = roc_auc_score(valid_y_true, valid_y_pred)
    print '\n- **Evaluation results of CNN BiLSTM siamese mixed model\n'
    print 'Precision:', precision, '\n'
    print 'Recall:', recall, '\n'
    print 'F1:', f1, '\n'
    print 'Accuracy:', accuracy, '\n'
    print 'Auc:', auc, '\n'


def siamese_categorical(level):
    config = Config()
    config.exp_name = 'siamese_categorical_' + level
    config.tensorboard_log_dir = os.path.join('experiments', config.exp_name, 'logs')
    config.checkpoint_dir = os.path.join('experiments', config.exp_name, 'checkpoints')
    config.embedding_path = 'custom_data/%s_level' % level
    if level == 'word':
        config.max_len = config.max_len_word
    else:
        config.max_len = config.max_len_char

    create_dirs([config.tensorboard_log_dir, config.checkpoint_dir])

    print('Create the data generator.')
    data_loader = SiameseDataLoader(config,
                                    train_file='datasets/train.csv',
                                    valid_file='datasets/valid.csv',
                                    level=level)

    print('Create the model.')
    siamese_categorical_model = SiameseCategoricalModel(config, data_loader.get_vocabulary())

    print('Create the trainer')
    trainer = SiameseModelTrainer(siamese_categorical_model.model,
                                  data_loader.get_train_data(),
                                  data_loader.get_valid_data(),
                                  config)

    print('Start training the model.')
    trainer.train()

    print('Start evaluate the model.')
    best_weights = os.path.join('experiments', config.exp_name, 'checkpoints', config.exp_name + '.hdf5')
    siamese_categorical_model.load(best_weights)
    valid_x_a, valid_x_b, valid_y = data_loader.get_valid_data()
    valid_results = siamese_categorical_model.model.predict([valid_x_a, valid_x_b])
    valid_y_pred = [valid_result[0] > 0.4 for valid_result in valid_results]
    valid_y_true = valid_y
    precision = precision_score(valid_y_true, valid_y_pred)
    recall = recall_score(valid_y_true, valid_y_pred)
    f1 = f1_score(valid_y_true, valid_y_pred)
    accuracy = accuracy_score(valid_y_true, valid_y_pred)
    auc = roc_auc_score(valid_y_true, valid_y_pred)
    print '\n- **Evaluation results of siamese categorical %s model**\n' % level
    print 'Precision:', precision, '\n'
    print 'Recall:', recall, '\n'
    print 'F1:', f1, '\n'
    print 'Accuracy:', accuracy, '\n'
    print 'Auc:', auc, '\n'


def siamese_categorical_mixed():
    config = Config()
    config.exp_name = 'siamese_categorical_mixed'
    config.tensorboard_log_dir = os.path.join('experiments', config.exp_name, 'logs')
    config.checkpoint_dir = os.path.join('experiments', config.exp_name, 'checkpoints')
    config.embedding_path_word = 'custom_data/word_level'
    config.embedding_path_char = 'custom_data/char_level'

    create_dirs([config.tensorboard_log_dir, config.checkpoint_dir])

    print('Create the data generator.')
    data_loader_word = SiameseDataLoader(config,
                                         train_file='datasets/train.csv',
                                         valid_file='datasets/valid.csv',
                                         level='word')

    data_loader_char = SiameseDataLoader(config,
                                         train_file='datasets/train.csv',
                                         valid_file='datasets/valid.csv',
                                         level='char')

    print('Create the model.')
    siamese_categorical_model = SiameseMixedCategoricalModel(config,
                                                             data_loader_word.get_vocabulary(),
                                                             data_loader_char.get_vocabulary())

    print('Create the trainer')
    trainer = SiameseMixedTrainer(siamese_categorical_model.model,
                                  data_loader_word.get_train_data(),
                                  data_loader_word.get_valid_data(),
                                  data_loader_char.get_train_data(),
                                  data_loader_char.get_valid_data(),
                                  config)

    print('Start training the model.')
    trainer.train()

    print('Start evaluate the model.')
    best_weights = os.path.join('experiments', config.exp_name, 'checkpoints', config.exp_name + '.hdf5')
    siamese_categorical_model.load(best_weights)
    valid_x_a_word, valid_x_b_word, valid_y_word = data_loader_word.get_valid_data()
    valid_x_a_char, valid_x_b_char, valid_y_char = data_loader_char.get_valid_data()
    assert valid_y_word == valid_y_char
    valid_results = siamese_categorical_model.model.predict([valid_x_a_word, valid_x_b_word,
                                                             valid_x_a_char, valid_x_b_char])
    valid_y_pred = [valid_result[0] > 0.4 for valid_result in valid_results]
    valid_y_true = valid_y_word
    precision = precision_score(valid_y_true, valid_y_pred)
    recall = recall_score(valid_y_true, valid_y_pred)
    f1 = f1_score(valid_y_true, valid_y_pred)
    accuracy = accuracy_score(valid_y_true, valid_y_pred)
    auc = roc_auc_score(valid_y_true, valid_y_pred)
    print '\n- **Evaluation results of siamese categorical mixed model**\n'
    print 'Precision:', precision, '\n'
    print 'Recall:', recall, '\n'
    print 'F1:', f1, '\n'
    print 'Accuracy:', accuracy, '\n'
    print 'Auc:', auc, '\n'


def ensemble():
    config = Config()
    config.exp_name = 'ensemble'
    config.tensorboard_log_dir = os.path.join('experiments', config.exp_name, 'logs')
    config.checkpoint_dir = os.path.join('experiments', config.exp_name, 'checkpoints')
    config.embedding_path_word = 'custom_data/word_level'
    config.embedding_path_char = 'custom_data/char_level'

    create_dirs([config.tensorboard_log_dir, config.checkpoint_dir])

    print('Create the data generator.')
    data_loader_word = SiameseDataLoader(config,
                                         train_file='datasets/train.csv',
                                         valid_file='datasets/valid.csv',
                                         level='word')
    data_loader_char = SiameseDataLoader(config,
                                         train_file='datasets/train.csv',
                                         valid_file='datasets/valid.csv',
                                         level='char')

    print('Create the base neural network model.')
    word_models = list()
    char_models = list()

    config.embedding_path = config.embedding_path_char
    config.max_len = config.max_len_char
    cnn_siamese_model_char = CNNSiameseModel(config, data_loader_char.get_vocabulary())
    char_models.append(cnn_siamese_model_char.build_model_without_compile())

    config.embedding_path = config.embedding_path_word
    config.max_len = config.max_len_word
    cnn_siamese_model_word = CNNSiameseModel(config, data_loader_word.get_vocabulary())
    word_models.append(cnn_siamese_model_word.build_model_without_compile())

    config.embedding_path = config.embedding_path_char
    config.max_len = config.max_len_char
    bilstm_siamese_model_char = BiLSTMSiameseModel(config, data_loader_char.get_vocabulary())
    char_models.append(bilstm_siamese_model_char.build_model_without_compile())

    config.embedding_path = config.embedding_path_word
    config.max_len = config.max_len_word
    bilstm_siamese_model_word = BiLSTMSiameseModel(config, data_loader_word.get_vocabulary())
    word_models.append(bilstm_siamese_model_word.build_model_without_compile())

    config.embedding_path = config.embedding_path_char
    config.max_len = config.max_len_char
    siamese_categorical_model_char = SiameseCategoricalModel(config, data_loader_char.get_vocabulary())
    char_models.append(siamese_categorical_model_char.build_model_without_compile())

    config.embedding_path = config.embedding_path_word
    config.max_len = config.max_len_word
    siamese_categorical_model_word = SiameseCategoricalModel(config, data_loader_word.get_vocabulary())
    word_models.append(siamese_categorical_model_word.build_model_without_compile())

    print('Create the ensemble neural network model.')
    ensemble_model = EnsembleModel(config,
                                   data_loader_word.get_vocabulary(),
                                   data_loader_char.get_vocabulary(),
                                   word_models,
                                   char_models)

    print('Create the trainer')
    trainer = SiameseMixedTrainer(ensemble_model.model,
                                  data_loader_word.get_train_data(),
                                  data_loader_word.get_valid_data(),
                                  data_loader_char.get_train_data(),
                                  data_loader_char.get_valid_data(),
                                  config)

    print('Start training the model.')
    trainer.train()

    print('Start evaluate the model.')
    best_weights = os.path.join('experiments', config.exp_name, 'checkpoints', config.exp_name + '.hdf5')
    ensemble_model.load(best_weights)
    valid_x_a_word, valid_x_b_word, valid_y_word = data_loader_word.get_valid_data()
    valid_x_a_char, valid_x_b_char, valid_y_char = data_loader_char.get_valid_data()
    assert valid_y_word == valid_y_char
    valid_results = ensemble_model.model.predict([valid_x_a_word, valid_x_b_word,
                                                  valid_x_a_char, valid_x_b_char])
    valid_y_pred = [valid_result[0] > 0.4 for valid_result in valid_results]
    valid_y_true = valid_y_word
    precision = precision_score(valid_y_true, valid_y_pred)
    recall = recall_score(valid_y_true, valid_y_pred)
    f1 = f1_score(valid_y_true, valid_y_pred)
    accuracy = accuracy_score(valid_y_true, valid_y_pred)
    auc = roc_auc_score(valid_y_true, valid_y_pred)
    print '\n- **Evaluation results of ensemble model**\n'
    print 'Precision:', precision, '\n'
    print 'Recall:', recall, '\n'
    print 'F1:', f1, '\n'
    print 'Accuracy:', accuracy, '\n'
    print 'Auc:', auc, '\n'


def bilstm_att_siamese(level):
    config = Config()
    config.exp_name = 'bilstm_att_siamese_' + level
    config.tensorboard_log_dir = os.path.join('experiments', config.exp_name, 'logs')
    config.checkpoint_dir = os.path.join('experiments', config.exp_name, 'checkpoints')
    config.embedding_path = 'custom_data/%s_level' % level
    if level == 'word':
        config.max_len = config.max_len_word
    else:
        config.max_len = config.max_len_char

    create_dirs([config.tensorboard_log_dir, config.checkpoint_dir])

    print('Create the data generator.')
    data_loader = SiameseDataLoader(config,
                                    train_file='datasets/train.csv',
                                    valid_file='datasets/valid.csv',
                                    level=level)

    print('Create the model.')
    bilstm_att_siamese_model = BiLSTMAttSiameseModel(config, data_loader.get_vocabulary())

    # best_weights = os.path.join('experiments', config.exp_name, 'checkpoints', config.exp_name + '.hdf5')
    # bilstm_att_siamese_model.load(best_weights)

    print('Create the trainer')
    trainer = SiameseModelTrainer(bilstm_att_siamese_model.model,
                                  data_loader.get_train_data(),
                                  data_loader.get_valid_data(),
                                  config)

    print('Start training the model.')
    trainer.train()

    print('Start evaluate the model.')
    best_weights = os.path.join('experiments', config.exp_name, 'checkpoints', config.exp_name + '.hdf5')
    bilstm_att_siamese_model.load(best_weights)
    valid_x_a, valid_x_b, valid_y = data_loader.get_valid_data()
    valid_results = bilstm_att_siamese_model.model.predict([valid_x_a, valid_x_b])
    valid_y_pred = [valid_result[0] < 0.65 for valid_result in valid_results]
    valid_y_true = valid_y
    precision = precision_score(valid_y_true, valid_y_pred)
    recall = recall_score(valid_y_true, valid_y_pred)
    f1 = f1_score(valid_y_true, valid_y_pred)
    accuracy = accuracy_score(valid_y_true, valid_y_pred)
    auc = roc_auc_score(valid_y_true, valid_y_pred)
    print '\n- **Evaluation results of BiLSTM Attention siamese %s model**\n' % level
    print 'Precision:', precision, '\n'
    print 'Recall:', recall, '\n'
    print 'F1:', f1, '\n'
    print 'Accuracy:', accuracy, '\n'
    print 'Auc:', auc, '\n'

    # output_error_lines(valid_y_pred, valid_y_true, valid_results, data_loader)
    #
    # valid_y_pred = post_processing(valid_y_pred, data_loader)
    # precision = precision_score(valid_y_true, valid_y_pred)
    # recall = recall_score(valid_y_true, valid_y_pred)
    # f1 = f1_score(valid_y_true, valid_y_pred)
    # accuracy = accuracy_score(valid_y_true, valid_y_pred)
    # auc = roc_auc_score(valid_y_true, valid_y_pred)
    # print '\n- **Evaluation results of BiLSTM Attention siamese %s model (post processing)**\n' % level
    # print 'Precision:', precision, '\n'
    # print 'Recall:', recall, '\n'
    # print 'F1:', f1, '\n'
    # print 'Accuracy:', accuracy, '\n'
    # print 'Auc:', auc, '\n'
